#coding=utf-8
import time

import traceback
from monthdelta import monthdelta
from datetime import datetime
import requests
from pymongo import MongoClient
from simplemysql import SimpleMysql

import Utils
import sys
from pandas import DataFrame
import pandas as pd
import json

reload(sys)
sys.setdefaultencoding('utf-8')


mgClient = MongoClient('121.40.54.235', 27017,
    username='root2',
    password='Dream2015',
    authSource="stockDataStore",
    authMechanism='SCRAM-SHA-1')
db = mgClient['stockDataStore']

mysqlDB = SimpleMysql(
    host="121.40.54.235",
    db="stock-grab",
    user="root",
    passwd="Dream2015",
    charset="utf8",
    autocommit=False,
    keep_alive=True  # try and reconnect timedout mysql connections?
)

top10_table_name = 'stock_top_shareholder'
summary_table_name = 'stock_shareholder_summary'


def transfer_to_period(period):
    period = period.encode("utf-8")
    '''
    period -> 2018年中报
    period -> 2005-10-23(股改前)
    '''
    '''
    2017年年报 -> 2017-12-31
    2017年三季报 -> 2017-09-30
    2017年中报 -> 2017-06-30
    2017年一季报 -> 2017-03-31
    '''
    map = {
        '年年报': '12-31',
        '年三季报': '09-30',
        '年中报': '06-30',
        '年一季报': '03-31',
    }
    year = period[0:4]
    key = period[4:]
    if map.has_key(key):
        month_day = map[key]
        #为什么要replace "－"， 要保持所有在mongo里面的日期格式一致
        return (year + "-" + month_day).replace("-", "");
    else:
        return period[0: 10].replace("-", '')

userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'


## -------------------mysql-----------------###
def top10_already_in_mysql(code, category, period, name):
    max_day_cursor = mysqlDB.query("select count(period) from " + top10_table_name + " where code =%s and category=%s and period =%s and name =%s", [code, category, period, name])
    count = max_day_cursor.fetchone()[0]
    if count != 0:
        return True
    else:
        return False

def summary_already_in_mysql(code, period):
    max_day_cursor = mysqlDB.query("select count(period) from " + summary_table_name + " where code = %s and  period = %s", [code, period])
    count = max_day_cursor.fetchone()[0]
    if count != 0:
        return True
    else:
        return False


def get_max_period_from_mysql(stock_code, category):
    max_day_cursor = mysqlDB.query("select max(period) from " + top10_table_name + " where code = %s and category= %s",
                                   [stock_code, category])
    max_period = max_day_cursor.fetchone()[0]
    return max_period

## -------------------mongo-----------------###
def top10_already_in_mongo(code, category, period, name):
    one = db[top10_table_name].find_one({'code': code, 'category': category, 'period': period, 'name': name}, {'code':1})
    if one:
        return True
    else:
        return False

def summary_already_in_mongo(code, period):
    one = db[summary_table_name].find_one({'code': code, 'period': period}, {'code': 1})
    if one:
        return True
    else:
        return False


def get_max_period_from_mongo(stock_code, category):
    max_period_cursor = db[top10_table_name].aggregate([
        {"$match": {"code": stock_code, "category": category}},
        {"$group": {"_id": '$code', "max": {"$max": "$period"}}}
    ])
    result = list(max_period_cursor)
    if len(result) > 1:
        Utils.logger.error("there is dirty data inside collection table: " + top10_table_name)
    if len(result) == 1:
        max = result[0]
        return max['max'];
    return None

def get_max_period_of_summary_from_mongo(stock_code):
    max_period_cursor = db[top10_table_name].aggregate([
        {"$match": {"code": stock_code}},
        {"$group": {"_id": '$code', "max": {"$max": "$period"}}}
    ])
    result = list(max_period_cursor)
    if len(result) > 1:
        Utils.logger.error("there is dirty data inside collection table: " + top10_table_name)
    if len(result) == 1:
        max = result[0]
        return max['max'];
    return None
#########################################################

def downoad_parse_top10_shareholder(stock_code, start='1990-01-01', end=datetime.now().strftime("%Y-%m-%d")):
    stock_suffix = ".SZ"  ##创业板和深圳中小板
    if stock_code.startswith("6"):  ##沪市主板
        stock_suffix = ".SH"
    url= "http://app2.jg.eastmoney.com/stock/F9/GetTop10HolderList?securityCode="+stock_code + stock_suffix +"&startDate="+start+"&endDate="+end+"&order=desc"
    response = requests.request("get", url, headers= {"user-agent": userAgent});
    response_content = response.content.decode("UTF-8");
    period_data_list = json.loads(response_content)
    records = []
    totals = []
    for period_data in period_data_list:
        period_str = period_data['Date']
        period = transfer_to_period(period_str)
        for r in period_data['Details']:
            if r['SHAREHDNAME'] == '合计':
                total = {}
                total['code'] = stock_code
                total['period'] = period
                total['num'] = long(float(r['SHAREHDNUM'].replace(',', '')))
                total['num_pct'] = float(r['SHAREHDNUMPER'])
                total['createTime'] = datetime.now()
                total['modifyTime'] = datetime.now()
                totals.append(total)
            else:
                record = {}
                record['code'] = stock_code
                record['period'] = period
                record['category'] = 'Top10'
                record['name'] = r['SHAREHDNAME']
                record['nature'] = r['SHAREHDNATURE']
                record['num'] = long(float(r['SHAREHDNUM'].replace(',', '')))
                if r['SHAREHDNUM_CHANGE'] == '不变' :
                    record['num_change'] = long(0)
                elif r['SHAREHDNUM_CHANGE'] == '新进' :
                    record['num_change'] = record['num']
                else :
                    record['num_change'] = long(float(r['SHAREHDNUM_CHANGE'].replace(',', '')))
                record['num_pct'] = float(r['SHAREHDNUMPER'])
                record['direction'] = r['DIRECTION']
                record['createTime'] = datetime.now()
                record['modifyTime'] = datetime.now()
                records.append(record)
    return {"records": records, "totals": totals}

def downoad_parse_top10_circulation_shareholder(stock_code, start='1990-01-01', end=datetime.now().strftime("%Y-%m-%d")):
    stock_suffix = ".SZ"  ##创业板和深圳中小板
    if stock_code.startswith("6"):  ##沪市主板
        stock_suffix = ".SH"
    url = "http://app2.jg.eastmoney.com/stock/f9/GetTop10CirculationShareHolder?securityCode=" + stock_code + stock_suffix + "&startDate="+start+"&endDate="+end+"&order=desc"
    response = requests.request("get", url, headers= {"user-agent": userAgent});
    response_content = response.content.decode("UTF-8");
    period_data_list = json.loads(response_content)
    records = []
    totals = []
    for period_data in period_data_list:
        period = transfer_to_period(period_data['report'])
        for r in period_data['records']:
            if r['SHAREHDNAME'] == '合计':
                total = {}
                total['code'] = stock_code
                total['period'] = period
                total['num'] = long(float(r['SHAREHDNUM'].replace(',', '')))
                total['num_pct'] = float(r['SHAREHDNUMPERG'])
                total['createTime'] = datetime.now()
                total['modifyTime'] = datetime.now()
                totals.append(total)
            else:
                record = {}
                record['code'] = stock_code
                record['period'] = period
                record['category'] = 'Top10Circulation'
                record['name'] = r['SHAREHDNAME']
                record['nature'] = r['SHAREHDNATURE']
                record['num'] = long(float(r['SHAREHDNUM'].replace(',', '')))
                # like '66,032,468'， 当没有变化的时候 '不变'
                if r['SHAREHDNUM_CHANGE2'] == '不变':
                    record['num_change'] = long(0)
                elif r['SHAREHDNUM_CHANGE2'] == '新进':
                    record['num_change'] = record['num']

                else:
                    record['num_change'] = long(float(r['SHAREHDNUM_CHANGE2'].replace(',', '')))

                record['num_pct'] = float(r['SHAREHDNUMPERG'])
                record['direction'] = r['DIRECTION']
                record['createTime'] = datetime.now()
                record['modifyTime'] = datetime.now()
                records.append(record)

    return {"records": records, "totals": totals}


def composeSummaryFromTotal(total, category, stock_code):
    category_field_name = 'top10_'
    if category != 'Top10':
        category_field_name = "top10_circulation_"
    summary = {
        'code': stock_code,
        'period': total['period'],
        (category_field_name + 'num'): total['num'],
        (category_field_name + 'num_pct'): total['num_pct'],
        'createTime': datetime.now(),
        'modifyTime': datetime.now()
    }
    return summary

def save_to_mysql(stock_code, category, records, totals):
    insert_count = 0
    insert_summary_count = 0
    for record in records:
        if not top10_already_in_mysql(stock_code, category, record['period'], record['name']):
            mysqlDB.insert(top10_table_name, record)
            insert_count = insert_count + 1

    for total in totals:
        summary = composeSummaryFromTotal(total, category, stock_code)
        if not summary_already_in_mysql(stock_code, total['period']):
            mysqlDB.insert(summary_table_name, summary)
            insert_summary_count = insert_summary_count + 1
        else:
            condition = ("code=%s AND period=%s", [stock_code, summary['period']])
            mysqlDB.update(summary_table_name, summary, condition)
    Utils.logger.info("count(record/summary) of " + category + " of stock" + stock_code
                      + " insert to mysql: " + str(insert_count) + "/" + str(insert_summary_count))

def save_to_mongo(stock_code, category, records, totals):
    insert_count = 0
    insert_summary_count = 0
    for record in records:
        if not top10_already_in_mongo(stock_code, category, record['period'], record['name']):
            db[top10_table_name].insert(record)
            insert_count = insert_count + 1

    for total in totals:
        summary = composeSummaryFromTotal(total, category, stock_code)
        if not summary_already_in_mongo(stock_code, total['period']):
            db[summary_table_name].insert(summary)
            insert_summary_count = insert_summary_count + 1
        else:
            db[summary_table_name].update_one({"code": stock_code, "period": summary['period']}, {"$set": summary})
    Utils.logger.info("count inserting to mongo of category " + category + " of stock" + stock_code
                      + " is " + str(insert_count) + "/" + str(insert_summary_count))


def collect_shareholder_to_mongo(stock_code, category):

    max_period = get_max_period_from_mongo(stock_code, category)
    if max_period is None:
        max_period = "1990-01-01"
    max_period_summary = get_max_period_of_summary_from_mongo(stock_code)
    if max_period_summary is None:
        max_period_summary = "1990-01-01"
    #取较小的日期
    if max_period > max_period_summary:
        max_period = max_period_summary

    if category == 'Top10':
        data = downoad_parse_top10_shareholder(stock_code, max_period)
    else:
        data = downoad_parse_top10_circulation_shareholder(stock_code, max_period)
    records = data['records']
    totals = data['totals']
    save_to_mongo(stock_code, category, records, totals)

def collect(stocks):
    for stock in stocks[:]:
        try:
            if Utils.bypass_stock(stock['code']):
                Utils.logger.info(stock['code'] + " bypassed ")
                continue
            Utils.logger.info("--->start to collect stock top shareholder " + stock["name"] + "-" + stock["code"])
            code = stock['code'];
            collect_shareholder_to_mongo(code, 'Top10')
            mysqlDB.commit()
            collect_shareholder_to_mongo(code, 'Top10Circulation')
            mysqlDB.commit()
            Utils.logger.info("--->success to collect stock top shareholder " + stock["name"] + "-" + stock["code"] + "\n")
        except Exception as es:
            traceback.print_exc()
            Utils.logger.error("--->fail to collect stock top shareholder " + stock["name"] + "-" + stock["code"] + "\n")

def main():
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^00"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    Utils.logger.info("CollectionShareHolders_eastmoney.py start to collect stock of exp ^00, count: " + str(len(stocks)))
    collect(stocks)
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^30"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    Utils.logger.info("CollectionShareHolders_eastmoney.py start to collect stock of exp ^30, count: " + str(len(stocks)))
    collect(stocks)
    stocks = Utils.quick_access_cursor(db['stock'].find({"code": {"$regex": "^60"}}, projection={"code": 1, "name": 1}, sort=[("code", 1)]));
    Utils.logger.info("CollectionShareHolders_eastmoney.py start to collect stock of exp ^60, count: " + str(len(stocks)))

    collect(stocks)

collect([{'code': '600009', 'name': 'zgpa'}])
#main()

Utils.logger.info("CollectionShareHolders_eastmoney.py exit")
